/* 
 * 数据处理模块
 * 1.去标签
 * 2.将一个文件合并成一行
 * 3.提取出文档的结构进行分析 标题-正文-url
 *
 */

#include <iostream>
#include <string>
#include <vector>
#include <fstream>
#include "../common/util.hpp"
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/operations.hpp>

const std::string g_input_path = "../data/input";
const std::string g_output_path = "../data/temp/raw_input";

struct DocInfo{
  std::string title;
  std::string content;
  std::string url;
};

bool EnumFile(const std::string& input_path,
  std::vector<std::string>* file_list){
  namespace fs = boost::filesystem;
  fs::path root_path(input_path);//根据字符串构造出path对象，能
                                 //对文件进行描述
  if(!fs::exists(root_path)){
    std::cout<<"input_path not exist! input_path="<<input_path<<std::endl;
    return false;
  }
  
  fs::recursive_directory_iterator end_iter;
  for(fs::recursive_directory_iterator iter(root_path);
        iter != end_iter; ++iter)
  {
    //剔除目录
    if(!fs::is_regular_file(*iter)) continue;
    //剔除其他文件拓展名文件
    if(iter->path().extension() != ".html") continue;
    file_list->push_back(iter->path().string());
  }

  return true;
}

bool ParseTitle(const std::string& html, std::string* title)
{ 
  //先查找<title>标签 ，再查找</title>
  size_t beg = html.find("<title>");
  if(beg == std::string::npos){
    std::cout<<"<title> is not found! "<<std::endl;
    return false;
  }
  size_t end = html.find("</title>");
  if(end == std::string::npos){
    std::cout<<"</title> not found!"<<std::endl;
    return false;
  }

  beg += std::string("<title>").size();
  if(beg >= end)
  {
    std::cout<<"beg > end"<<std::endl;
    return false;
  }
  std::cout<<"beg:"<<beg<<" "<<"end - beg:"<<end<<std::endl;
  *title = html.substr(beg, end - beg);
  return true;
}

bool ParseContent(const std::string& html,
      std::string* content)
{
  bool is_content = true;
  for(auto c : html)
  {
    if(is_content){
      if(c == '<')
      {
        is_content = false;
      }
      else{
        if(c == '\n') c = ' ';
        content->push_back(c);
      }
    }else{
      if(c == '>') is_content = true;
    } 
  }
  return true;
}

//Boost 文档有统一的前缀
bool ParseUrl(const std::string& file_path,
    std::string* url){
  std::string prefix = "https://www.boost.org/doc/libs/1_53_0/doc/";
  std::string tail = file_path.substr(g_input_path.size());
  *url = prefix + tail; 
  return true;
}


bool ParseFile(const std::string& file_path, DocInfo* doc_info)
{
  //1.打开文件，读取内容
  std::string html;
  bool ret = FileUtill::Read(file_path, &html);
  if(!ret){
    std::cout<<"Read file failed ! path="<<file_path<<std::endl;
    return false;
  }

  //2.解析标题
  ret = ParseTitle(html, &doc_info->title);
  if(!ret)
  {
    std::cout<<"ParseTitle Failed! path="<<file_path<<std::endl;
    return false;
  }
  //std::cout<<"DEBUG TITLE:"<<doc_info->title<<std::endl;
  //3.解析正文，去除html标签
  ret = ParseContent(html, &doc_info->content);
  if(!ret)
  {
    std::cout<<"ParseContent Failed! path="<<file_path<<std::endl;
    return false;
  }
  //4.解析url
  ret = ParseUrl(file_path, &doc_info->url);
  if(!ret)
  {
    std::cout<<"ParseUrl Failed! path="<<file_path<<std::endl;
    return false;
  }
  return true;
} 

void WriteOutput(const DocInfo& doc_info, std::ofstream& file){
  std::string line = doc_info.title + "\3" + doc_info.url + "\3" +doc_info.content + "\n";
  file.write(line.c_str(), line.size());
}

int main()
{
  //1.枚举出输入路径中所有的html路径
  std::vector<std::string> file_list;
  bool ret = EnumFile(g_input_path, &file_list);
  if(ret == false) {
    std::cout<<"Enum Failed!"<<std::endl;
   return 1;
  }

  std::ofstream output_file(g_output_path.c_str());//写操作的文件对象
  if(!output_file.is_open()){
    std::cout<<"Open Output_File Failed"<<std::endl;
    return 1;
  }
  
  //2.提取文件的标题/正文/url，去标签
  for(const auto& file_path : file_list)
  {
    //输入的是当前要解析的文件路径
    //输出的是解析后得到的DocInfo结构
    DocInfo info;
    ret = ParseFile(file_path, &info);
    if(!ret) {
      std::cout<<"Parse File Failed!  file_path="<<file_path<<std::endl;
      continue;
    }
    WriteOutput(info, output_file);
  }
  //3.将分析的结果按照一定的形式写入到输出文件中
  output_file.close();
  std::cout<<"Finish"<<std::endl; 
  return 0;
}


